package com.example.springboot_demo.service;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.example.springboot_demo.mappers.AllSchoolMapper;
import com.example.springboot_demo.mappers.GovDocumentMapper;
import com.example.springboot_demo.pojo.AllSchool;
import com.example.springboot_demo.pojo.GovDocument;
import io.netty.util.internal.NativeLibraryLoader;
import lombok.SneakyThrows;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.Get;
import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.omg.CORBA.OBJ_ADAPTER;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.web.util.HtmlUtils;

import java.io.IOException;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.*;
import java.util.stream.Collectors;

/**
 * 国务院
 */
@Component
public class GovService {

    @Autowired
    private GovDocumentMapper govDocumentMapper;

    List<String> userAgent = Arrays.asList("Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;AvantBrowser)",
            "Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;360SE)",
            "Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;Trident/4.0;SE2.XMetaSr1.0;SE2.XMetaSr1.0;.NETCLR2.0.50727;SE2.XMetaSr1.0)",
            "Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;TheWorld)",
            "Mozilla/4.0(compatible;MSIE7.0;WindowsNT5.1;TencentTraveler4.0)",
            "Opera/9.80(Macintosh;IntelMacOSX10.6.8;U;en)Presto/2.8.131Version/11.11",
            "Mozilla/5.0(WindowsNT6.1;rv:2.0.1)Gecko/20100101Firefox/4.0.1",
            "Mozilla/5.0(compatible;MSIE9.0;WindowsNT6.1;Trident/5.0",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36",
            "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)",
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 UBrowser/4.0.3214.0 Safari/537.36",
            "Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1",
            "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
            "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
            "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
            "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
            "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
            "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
            "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
            "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
            "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
            "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
            "PostmanRuntime/7.28.4",
            "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52");


    public Set<String> getFromDb() {
        List<String> strings = govDocumentMapper.queryIds();
        return new HashSet<>(strings);
    }


    public List<Map<String, Object>> getPage(String keyWord,int year) {
        //根据数字年份，得到当年1月1日0时0分0秒的时间戳和12月31日23时59分59秒的时间戳
        // 获取当年1月1日0时0分0秒的时间戳
        LocalDateTime startDateTime = LocalDateTime.of(year, 1, 1, 0, 0, 0);
        long startTimeStamp = startDateTime.toEpochSecond(ZoneOffset.UTC) * 1000;
        // 获取当年12月31日23时59分59秒的时间戳
        LocalDateTime endDateTime = LocalDateTime.of(year, 12, 31, 23, 59, 59);
        long endTimeStamp = endDateTime.toEpochSecond(ZoneOffset.UTC) * 1000;
        Set<String> fromDb = getFromDb();
        int page = 1;
        String url = "https://sousuoht.www.gov.cn/athena/forward/2B22E8E39E850E17F95A016A74FCB6B673336FA8B6FEC0E2955907EF9AEE06BE";
        // 初始化 HashMap
        Map<String, Object> map = new HashMap<>();
        // 逐个放入键值对
        map.put("code", "17da70961a7");
        map.put("historySearchWords", new String[]{});
        map.put("dataTypeId", "14");
        map.put("orderBy", "related");
        map.put("searchBy", "all");
        map.put("appendixType", "");
        map.put("granularity", "CUSTOM");
        map.put("trackTotalHits", true);
        map.put("beginDateTime", startTimeStamp);
        map.put("endDateTime", endTimeStamp);
        map.put("isSearchForced", 0);
        map.put("filters", new String[]{});
        map.put("pageSize", 10);
        Map<String, Object> customFilter = new HashMap<>();
        customFilter.put("operator", "and");
        customFilter.put("properties", new String[]{});
        map.put("customFilter", customFilter);
        map.put("searchWord", keyWord);
        while (page<=15) {
            map.put("pageNo", page);
            //创建post请求对象
            HttpPost post = new HttpPost(url);
            String resultString = "";
            try {
                //添加参数
                post.setEntity(new StringEntity(JSON.toJSONString(map), "UTF-8"));
                post.setHeader("Content-type", "application/json");
                //添加请求头参数
                post.addHeader("User-Agent", userAgent.get(RandomUtils.nextInt(0, userAgent.size() - 1)));
                post.addHeader("athenaAppName", "%E5%9B%BD%E7%BD%91%E6%90%9C%E7%B4%A2");
                post.addHeader("athenaAppKey", "YszQ8rwvYp9dzqmF9AfZMzQlyGzeOUg7uflT0hdlkah36dr%2BbznLhsQw1VkHtCpYdcrDs9dQQB5LInZWnuVlDjoe9r4nBkjQc1S5L8G0Zvyv4PrOdYliyfs8PUo%2BlHRXJNX0a60PLnYqDbGyAUmJlh2NsDvCEXxm%2FnwfSh3UnCY%3D");
                post.addHeader("Accept", "application/json, text/plain, */*");
                post.addHeader("Accept-Encoding", "gzip, deflate, br");
                post.addHeader("Connection", "keep-alive");


                CloseableHttpClient client = HttpClients.createDefault();
                //启动执行请求，并获得返回值
                CloseableHttpResponse response = client.execute(post);
                //得到返回的entity对象
                HttpEntity resBody = response.getEntity();
                //把实体对象转换为string
                resultString = EntityUtils.toString(resBody, "UTF-8");
            }catch (Exception e) {
                e.printStackTrace();
                continue;
            }
            page++;
            Map resMap = null;
            try {
                resMap = JSON.parseObject(resultString, Map.class);
            } catch (Exception e) {
                e.printStackTrace();
                continue;
            }
            Map<String, Object> result = (Map<String, Object>) resMap.get("result");
            Map<String, Object> data = (Map<String, Object>) result.get("data");
            Map<String, Object> middle = (Map<String, Object>) data.get("middle");
            Map<String, Object> pager = (Map<String, Object>) data.get("pager");
            List<Map<String, Object>> list = (List<Map<String, Object>>) middle.get("list");
            if (CollectionUtils.isEmpty(list)) {
                break;
            }
            Integer  total = (Integer) pager.get("total");
            if (page > total / 10 + 1) {
                break;
            }
            System.out.println("当前年度："+year+", 关键词："+keyWord+", 总条数:"+pager.get("total")+", 总页数：" + pager.get("pageCount") + ", 当前页数：" + pager.get("pageNo"));
            for (Map<String, Object> item : list) {
                if (fromDb.contains(item.get("documentId"))) {
                    continue;
                }
                GovDocument content = getContent(item);
                //插入数据库
                if (content != null) {
                    try {
                        govDocumentMapper.insert(content);
                        System.out.println("插入成功:"+content.getTitle());
                    }catch (Exception e) {
                        e.printStackTrace();
                    }
                }

            }
            try {
                Thread.sleep(2000);
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
        }
        return null;
    }

    public GovDocument getContent(Map<String, Object> map) {
        HashMap<String, String> heards = new HashMap<>();
        //添加请求头参数
        heards.put("User-Agent", "userAgent.get(RandomUtils.nextInt(0, userAgent.size() - 1))");
        heards.put("Accept-Encoding", "gzip, deflate, br");
        heards.put("Connection", "keep-alive");
        heards.put("Accept", "application/json, text/plain, */*");
        heards.put("Accept-Encoding", "gzip, deflate, br");

        String url = (String) map.get("url");
        if (url == null) {
            return null;
        }
        Document document = null;
        GovDocument govDocument = new GovDocument();
        govDocument.setUrl(url);
        govDocument.setUniqeId((String) map.get("documentId"));
        govDocument.setPtime((String) map.get("time"));
        try {
            document = Jsoup.connect(url).headers(heards).get();
            // 头部。选择所需的元素并提取文字
            Elements head = document.select("div.pchide.abstract.mxxgkabstract *");
            for (int i = 0;  i< head.size(); i++) {
                String text = head.get(i).text();
                if (i == 1) {
                    govDocument.setIndexStr(text);
                } else if (i == 3) {
                    govDocument.setCategory(text);
                } else if (i == 5) {
                    govDocument.setSector(text);
                } else if (i == 7) {
                    govDocument.setCtime(text);
                } else if (i == 9) {
                    govDocument.setTitle(text);
                }else if (i == 11) {
                    govDocument.setFileNumber(text);
                } else if (i == 13) {
                    govDocument.setPtime(text);
                }else if (i == 15) {
                    govDocument.setMainWord(text);
                }else if (i == 17) {
                    govDocument.setRemark(text);
                }
            }
            if (head.isEmpty()) {
                head = document.select("ul.mobileTable span");
                for (int i = 0; i < head.size(); i++) {
                    String text = head.get(i).text();
                    if (i == 1) {
                        govDocument.setTitle(text);
                    } else if (i == 3) {
                        govDocument.setSector(text);
                    } else if (i == 5) {
                        govDocument.setFileNumber(text);
                    } else if (i == 7) {
                        govDocument.setSource(text);
                    } else if (i == 9) {
                        govDocument.setCategory(text);
                    }else if (i == 11) {
                        govDocument.setType(text);
                    }else if (i == 13) {
                        govDocument.setCtime(text);
                    }else if (i == 15) {
                        govDocument.setMainWord(text);
                    }else if (i == 17) {
                        govDocument.setRemark(text);
                    }
                }
            }
            //文章内容处理
            Element content = document.select("table.pages_content").first();
            if (content == null) {
                content = document.select("div.pages_content").first();
            }
            if (content!= null) {
                String text = content.text();
                StringBuilder stringBuilder = new StringBuilder();
                if (text.contains("兼并")) {
                    stringBuilder.append("兼并,");
                }else {
                    stringBuilder.append("--,");
                }
                if (text.contains("收购")) {
                    stringBuilder.append("收购,");
                }else {
                    stringBuilder.append("--,");
                }
                if (text.contains("并购")) {
                    stringBuilder.append("并购,");
                }else {
                    stringBuilder.append("--,");
                }
                if (text.contains("重组")) {
                    stringBuilder.append("重组");
                }else {
                    stringBuilder.append("--");
                }
                String tag = stringBuilder.toString();
                if (StringUtils.equals("--,--,--,--",tag)) {
                    return null;
                } else {
                    if (StringUtils.isBlank(govDocument.getTitle())) {
                        govDocument.setTitle((String) map.get("title_no_tag"));
                    }
                    govDocument.setTag(tag);
                    govDocument.setContent("<!doctype html>\n" +
                            "<html>\n" +
                            head.outerHtml() +
                            content.outerHtml() +
                            "</html>"
                    );
                    return govDocument;
                }

            }

        } catch (IOException e) {
            e.printStackTrace();
        }
        return null;
    }

}
